{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导入相应的库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Intstalled\\Anaconda2\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import seaborn as sns\n",
    "\n",
    "from matplotlib import pyplot\n",
    "\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "from sklearn.linear_model import RidgeCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import  GridSearchCV\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "import xgboost\n",
    "\n",
    "\n",
    "from xgboost import XGBClassifier\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = \"C:/Users/Admin/Desktop/69-56-1-1518155113/week3/code/data/\"\n",
    "\n",
    "#dtrain = xgb.DMatrix(dpath + 'agaricus.txt.train')\n",
    "\n",
    "#data = xgb.DMatrix(data_path +\"RentListingInquries_FE_train.bin\")\n",
    "\n",
    "#dtrain = xgb.DMatrix(data_path + 'RentListingInquries_FE_train.bin')\n",
    "\n",
    "#dtest = xgb.DMatrix(data_path + 'RentListingInquries_FE_test.bin')\n",
    "\n",
    "data = pd.read_csv(data_path +\"RentListingInquries_FE_train.csv\")\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 分割数据，因为数据量太大，分割出三分之一左右作为训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "y_train = data['interest_level']\n",
    "\n",
    "\n",
    "x_train = data.drop(['interest_level'],axis=1)\n",
    "\n",
    "train_X,test_X, train_y, test_y = train_test_split(x_train,  \n",
    "                                                   y_train,  \n",
    "                                                   test_size = 0.3,  \n",
    "                                                   random_state = 0)  \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#直接调用xgboost内嵌的交叉验证（cv），可对连续的n_estimators参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证\n",
    "def modelfit(alg, X_train, y_train, cv_folds=5, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3\n",
    "    \n",
    "    #直接调用xgboost，而非sklarn的wrapper类\n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  \n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    #train_predprob = alg.predict_proba(X_train)\n",
    "    #logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "   #Print model report:\n",
    "   # print (\"logloss of train :\" )\n",
    "   # print logloss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 获取弱学习器的个数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#params = {\"objective\": \"multi:softprob\", \"eval_metric\":\"mlogloss\", \"num_class\": 9}\n",
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb1, train_X, train_y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'base_score': 0.5,\n",
       " 'booster': 'gbtree',\n",
       " 'colsample_bylevel': 0.7,\n",
       " 'colsample_bytree': 0.8,\n",
       " 'gamma': 0,\n",
       " 'learning_rate': 0.1,\n",
       " 'max_delta_step': 0,\n",
       " 'max_depth': 5,\n",
       " 'min_child_weight': 1,\n",
       " 'missing': None,\n",
       " 'n_estimators': 191,\n",
       " 'nthread': 1,\n",
       " 'objective': 'multi:softprob',\n",
       " 'reg_alpha': 0,\n",
       " 'reg_lambda': 1,\n",
       " 'scale_pos_weight': 1,\n",
       " 'seed': 3,\n",
       " 'silent': 1,\n",
       " 'subsample': 0.3}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb1.get_xgb_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 画出结果曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xl8XXWd//HX596brc3WJmlpm+60\nlYKspYALFAEFlEVUpIIjuPBjRn4qjuMPZxyHwWHcRkcYGREREEUquCAqKsouspWlSFtbShea7lua\nNs1+P78/vifJbZq0aZubc9P7fj4e55Gz3XM+OTe573u+ZzN3R0REBCARdwEiIpI7FAoiItJFoSAi\nIl0UCiIi0kWhICIiXRQKIiLSRaEgksHM/tnMbou7DpG4KBSGGDMrNbOVZvahjHFlZvaGmb0/Y9ws\nM/uNmW0zs3ozW2RmN5jZiGj65WbWYWY7o265mf19lmufY2Z12VzH/uitHnf/T3f/eJbWt9LMzszG\nsrNhsN6vobZdDnUKhSHG3XcCVwI3mllNNPrrwHx3/xmAmb0FeAx4CniTu1cCZwPtwDEZi3va3Uvd\nvRR4P/B1MztucH4T2R9mloq7BskT7q5uCHbAncA9wBxgCzAmY9qfgf/Zx+svB/7cY9xzwIcyhs8H\nFgL1hJA5ImPaEdG4+mie8zOmnQssAnYAa4DPAcOBJiAN7Iy6sX38XjcDv41e/ywwtR/b403AH4Gt\nwBLg4gOpB7gO+HH0ukmAA1cAq4FtwFXAicAr0e/+nYz1TAUeid6PzcDdQGU07UfRupqidX2+H9t4\nJfD/onW1AKloeE30uywBzuhlW5wMrAeSGePeC7wS9c8G5gMNwAbgW31s0zlAXR/TKoC7gE3AKuCL\nQCKalgS+GW2DFcDV0XZM9bGslcCZfUz7BLAsel8f6PybAQz4b2AjsD3aRkf19X7H/f86lLrYC1B3\ngG8cjADWRf94V2SMHw50AHP28frLyQiF6IOuHpgeDU8HGoGzgALg89E/Z2E0vAz452j4HdE/4Izo\nteuAt2fUeXzU3+eHTEYdd0YfALOjD8G7gXn7eM1wwof2FdFrjo+2y5H7Ww+9h8ItQDHwTqAZuB8Y\nBYyLPpROi+Y/PNpeRUAN8ATw7Yxl7/bht7dtnDH/y8B4oASYEf2eYzPq6zUwgdeBszKG7wOujfqf\nBj4c9ZcCJ/exjD7fL0Ig/Aooi+pYCnwsmnYV4UO5Ntref+IAQiH6u9ocvZ9FwP8AT0TT3gW8AFQS\nAuIIoi9Gfb3f6vrXqfloiHL3bYRvmMOAX2RMGkFoFlzfOcLMvh4dV2g0sy9mzHtyNH4nYS/hR8Br\n0bQPAr919z+6exvwX4QPprcQvomWAl9191Z3fwT4DTA3em0bMNPMyt19m7u/uJ+/3i/c/Tl3byeE\nwrH7mP89wEp3v8Pd26P1/ZzQJDYQ9XzZ3Zvd/SHCh/g97r7R3dcATwLHAbj7smh7tbj7JuBbwGl7\nWe7etnGnm9x9tbs3EcK+KPpdCtx9pbu/3sey7yF6P8ysjPDt+Z6M7XG4mVW7+053f2Z/NoaZJaPa\nv+DuO9x9JWHP4MPRLBcDN7p7XfR3+tX9WX6GS4Hb3f1Fd28BvgCcYmaTot+hjLCHaO6+2N3XZfx+\nB/N+5zWFwhBlZpcRvqH9CfhaxqRthGaKMZ0j3P3zHo4r/JLwTbrTM+5e6eGYwmHAkcB/RtPGEpoF\nOpeRJnxLHRdNWx2N67QqmgbwPsKH0Coze9zMTtnPX299Rv8uQgDtzUTgpCjg6s2snvCBctgA1bMh\no7+pl+FSADMbZWbzzGyNmTUAPwaq97LcvW3jTqszpi8DPkPYm9kYrWtsH8v+CXCRmRUBFwEvunvn\nuj5G2Ev5m5k9b2bv2UuNvakm7CGuyhiX+f6Pzay7R//+6Ll9dhKa5sZFX0S+Q2hq3GBmt5pZeTTr\nwb7feU2hMASZ2ShCe+ongP8DXGxmpwK4eyOhHf6i/Vmmu28gfLs+Lxq1lvBh27lOIzRjrImmjTez\nzL+fCdE03P15d7+A0MRyP3Bv52r2p6b9sBp4PAq4zq7U3f9+kOv5SrTMo929HLiM0LTRqef69raN\ne32Nu//E3d8Wvc7Z/QtB5nyLCB+o5wAfIoRE57TX3H0uYXt8DfiZmQ3v/6/JZsK38YkZ47ref0Lz\nTW3GtPH7sexMPbfPcKCK7r+zm9z9BMKXmenAP0Xj+3q/pR8UCkPTd4D73f3RaJf588D3o2+FRMMf\nNbNrowDBzGqByX0t0MyqCAcjF0aj7gXebWZnmFkB8I+Eg51/IYROI/B5MyswszmEMJlnZoVmdqmZ\nVURNIg2EZg8I37CrzKxigLZDp98A083sw1E9BWZ2opkdMcj1lBEOIteb2TiiD6kMG4ApGcN728Z7\nMLMZZvaO6H1uJuyldPQ2b+QnwKeAUwnHFDqXc5mZ1UR7JvXR6D6XY2bFmR1hT/Re4IbodOiJwGcJ\ne0adv9enzWycmVUSDo7vS0GP9aSi+q8ws2Oj3/k/gWfdfWX0/p4UbbfGaHt07OP9lv6I+6CGuv3r\ngAsJ36Aqe4x/GLghY/gk4EHCP3098CpwA1AVTb+c8M/SeebNRkKb86iMZbyXcMBwO/A40YHbaNqR\n0bjt0TzvjcYXAr8nNGM1AM8Db8t43e2EJoB6+j776D8yhuewj4PT0XwzCGcsbYqW/wjhWMR+1UPv\nB5pTGfPXkXEQn/BB+MWMbfJCtD1fJnzI12XMewHwRrSuz/VjG69k9wPTRxOO/ewgHIz/TW/bMGP+\nCYQP8N/2GP/j6P3eSfgScGEfr58T/f49u8MJx65+HG3v1cCX6D77KEXYk91COPvoGsKehfWxnpW9\nrOM/omlXEQ6ad/6+tdH4MwhnHO2k+0yv0n293+r23Vm0gUVEssLMzgFucfeJ+5xZYqfmIxEZUGZW\nYmbnmlkqakb7N8JJDjIEaE9BhgQzezvwu96meTh7SnKEmQ0jNIW9iXDc47fAp929IdbCpF8UCiIi\n0kXNRyIi0mXI3WSrurraJ02aFHcZIiJDygsvvLDZ3Wv2Nd+QC4VJkyYxf/78uMsQERlSzGzVvudS\n85GIiGRQKIiISBeFgoiIdFEoiIhIF4WCiIh0USiIiEgXhYKIiHTJm1BYvXUXf3h1Dbqth4hI3/Im\nFB7432s5874jaWxqirsUEZGclTehcMrMSSTN2bZxzb5nFhHJU3kTCgUV4RnuO7asi7kSEZHclTeh\nMGxECIVdWxUKIiJ9yZtQqKgeC0Brw4aYKxERyV35Ewo1IRQ6dmyMuRIRkdyVN6FQUFJOE0VY46a4\nSxERyVl5EwoA9VZBqmlz3GWIiOSsvAqFnamRlLRuibsMEZGclVeh0FRUxfD2bXGXISKSs/IqFNqL\nR1LRUR93GSIiOSuvQiE9bBQjaKC5tS3uUkREclLWQsHMbjezjWb2ah/TzcxuMrNlZvaKmR2frVo6\nJUtrSFmaLZvWZ3tVIiJDUjb3FO4Ezt7L9HOAaVF3JfDdLNYCQEHlaAAaNq/N9qpERIakrIWCuz8B\nbN3LLBcAd3nwDFBpZmOyVQ9ASWW41UXjNu0piIj0Js5jCuOA1RnDddG4PZjZlWY238zmb9p04Bef\nlVWFq5pb6hUKIiK9iTMUrJdxvT4Bx91vdfdZ7j6rpqbmgFdYURMyp71Bt7oQEelNnKFQB4zPGK4F\nstrYX1RaRTsJrFGhICLSmzhD4QHg76KzkE4Gtrt7du9rnUiw1SvYsqEuq6sRERmqUtlasJndA8wB\nqs2sDvg3oADA3W8BHgTOBZYBu4ArslVLpsbCKsbbjsFYlYjIkJO1UHD3ufuY7sAns7X+vuwqqqG8\nUQeaRUR6k1dXNAO0DRtNVXoL6XSvx7RFRPJa3oWCl42hyhrY0rAz7lJERHJO3oVCQWU4LXXr+jdi\nrkREJPfkXSiUVNUC0LBx9T7mFBHJP3kXChWjJgDQtFWhICLSU96FQuXoEArt9bopnohIT3kXCqnS\nalpJYTuye52ciMhQlHehgBnbElUUNm2IuxIRkZyTf6EA7CioZnjLgd9tVUTkUJWXodBcMprKjs1x\nlyEiknPyMhQ6ho+m2rfR1NoRdykiIjklL0MhUT6WUmtmw2Y1IYmIZMrLUPjtqvBz2/qVsdYhIpJr\n8jIULi/+MwANG1bFXImISG7Jy1AY+aFbAWjZovsfiYhkystQKBwxjg4SWL1CQUQkU16GAskCtiWq\nKGpcE3clIiI5JT9DAdheNIayZt3qQkQkU96GQkvpOEalN9LekY67FBGRnJG3oeAVtYxmK+u26Qls\nIiKd8jYUiqomk7I0G9esjLsUEZGckbehUHbYZAAa1i+LuRIRkdyRt6EwYuzhALRs1gVsIiKd8jYU\nCkeGJ7B5vR7LKSLSKW9DgYJitiVGULRToSAi0il/QwGoS1dT3KhnNYuIdMrrUChMGmPZRHObnqsg\nIgJZDgUzO9vMlpjZMjO7tpfpE83sYTN7xcweM7PabNazhylzGGebqdu8fVBXKyKSq7IWCmaWBG4G\nzgFmAnPNbGaP2f4LuMvdjwauB76SrXp6U3LY4RRYB+tXvz6YqxURyVnZ3FOYDSxz9+Xu3grMAy7o\nMc9M4OGo/9FepmfViNoZAOxYu3QwVysikrOyGQrjgMxTe+qicZkWAO+L+t8LlJlZVc8FmdmVZjbf\nzOZv2jRwj9AsGzMdgLbN2lMQEYHshoL1Ms57DH8OOM3MXgJOA9YA7Xu8yP1Wd5/l7rNqamoGrsLS\nw2ihiFT9yoFbpojIEJbK4rLrgPEZw7XAbud/uvta4CIAMysF3ufug3fUN5Fga+EYSnfpWgUREcju\nnsLzwDQzm2xmhcAlwAOZM5hZtZl11vAF4PYs1tOrxtKJjG5fS0u7TksVEclaKLh7O3A18AdgMXCv\nuy80s+vN7PxotjnAEjNbCowGbshWPX3WWTmJCbaRuq27BnvVIiI5J5vNR7j7g8CDPcZ9KaP/Z8DP\nslnDvhSNPpyS5a2sW7OCqaOOjrMUEZHY5fUVzQCV48JpqT/9/RMxVyIiEr+8D4XyseG01NNqdsRc\niYhI/PI+FKgYTxsFJLfqWgUREYVCMsXW4loqd63AvedlFCIi+UWhADSVT2Viuo5NO1viLkVEJFYK\nBSA5agYTbQPL1m6JuxQRkVgpFIDy8UeSNGfjykVxlyIiEiuFAiEUAJrWKhREJL8pFACrnk4aI7Hl\ntbhLERGJlUIBoHAY9QWHUbFzuc5AEpG8plCILO0YwwSdgSQieU6hEJkw4zim2FoWr6mPuxQRkdgo\nFCKVE99MsbWxevniuEsREYmNQiEybPwxADSvfjnmSkRE4qNQ6DTqCNIkKNiiPQURyV8KhU4FJWwr\nmcCYpmXsat3jMdEiInlBoZChreoI3mRv8Lf1uo22iOQnhUKGkvHHMCGxidfeWBt3KSIisVAoZCif\ndCwAf3z00ZgrERGJh0Ihg40+CoDjitbEXImISDwUCpkqamlOljFixxKaWjvirkZEZNApFDKZsavq\nSI60Fby6dnvc1YiIDLp9hoKZTTWzoqh/jpl9yswqs19aPIomzuIIW8UrKzfGXYqIyKDrz57Cz4EO\nMzsc+AEwGfhJVquK0fBJsyi0Dja//lLcpYiIDLr+hELa3duB9wLfdvdrgDHZLStGY48HILFOoSAi\n+ac/odBmZnOBjwC/icYVZK+kmFVOoKmgkonNf2Pjjua4qxERGVT9CYUrgFOAG9x9hZlNBn6c3bJi\nZEbrqGN5c2I5l37/2birEREZVPsMBXdf5O6fcvd7zGwEUObuX+3Pws3sbDNbYmbLzOzaXqZPMLNH\nzewlM3vFzM49gN9hwA2fPIvpVscZU0vjLkVEZFD15+yjx8ys3MxGAguAO8zsW/14XRK4GTgHmAnM\nNbOZPWb7InCvux8HXAL87/7+AtmQGj+LpDn1y5+PuxQRkUHVn+ajCndvAC4C7nD3E4Az+/G62cAy\nd1/u7q3APOCCHvM4UN65HiA3bjpUeyIAlVteZmeL7pgqIvmjP6GQMrMxwMV0H2juj3HA6ozhumhc\npuuAy8ysDngQ+L+9LcjMrjSz+WY2f9OmTftRwgEaXs2ussmcYEt5cdW27K9PRCRH9CcUrgf+ALzu\n7s+b2RTgtX68znoZ5z2G5wJ3unstcC7wIzPboyZ3v9XdZ7n7rJqamn6s+uAVTDqZ4xNLeW75lkFZ\nn4hILujPgeb73P1od//7aHi5u7+vH8uuA8ZnDNeyZ/PQx4B7o+U+DRQD1f0pPNsKJp1Cle3g8aef\njrsUEZFB058DzbVm9ksz22hmG8zs52ZW249lPw9MM7PJZlZIOJD8QI953gDOiNZzBCEUBqF9qB8m\nnAzAEe2LaNRxBRHJE/1pPrqD8GE+lnBM4NfRuL2KroK+mtD0tJhwltFCM7vezM6PZvtH4BNmtgC4\nB7jc3Xs2McWjahpthZWcYEt5doWakEQkP6T6MU+Nu2eGwJ1m9pn+LNzdHyQcQM4c96WM/kXAW/uz\nrEGXSJCYcBKzX3uFHy7dzDveNDruikREsq4/ewqbzewyM0tG3WVAXnx1Tk45lcm2jkVLFsddiojI\noOhPKHyUcDrqemAd8H7CrS8OfVPmADB+2/PUbdsVaykiIoOhP2cfveHu57t7jbuPcvcLCReyHfpG\nzaS9pJq3Jl/lstt0HyQROfQd6JPXPjugVeSqRILk1NM4LbmQKdXD465GRCTrDjQUersw7ZBkU+ZQ\nxTbWv65bXojIoe9AQyE3ThsdDFPmAHCKL+CJpblxCYWISLb0GQpmtsPMGnrpdhCuWcgPlRPw6hmc\nVfAKDy1cH3c1IiJZ1WcouHuZu5f30pW5e3+ubzhk2PR3cQKLeOSV5bS2p+MuR0Qkaw60+Si/TD+b\nAto5hVd48jU1IYnIoUuh0B/jT8KLKzincAG/ejk3HvkgIpINCoX+SKaww8/izNTL/GnROt0gT0QO\nWQqF/ppxDqXt25jZvpg/Ld4QdzUiIlnRn1tn93YW0urodtpTBqPInDD9XXiyiPNSz/GlXy2MuxoR\nkazoz57Ct4B/Itw2uxb4HPB9wjOXb89eaTmmqAybdhYXFb/AzuYW1m9vjrsiEZEB159QONvdv+fu\nO9y9wd1vBc51958CI7JcX26ZeSFlbZs4ltf4+Yt1cVcjIjLg+hMKaTO72MwSUXdxxrT8ubIZYMbZ\nkCzioyMWcN/81eTK84BERAZKf0LhUuDDwMao+zBwmZmVEJ6slj+KymD6Ozmj40lWb9nBuTc+GXdF\nIiIDap9XJrv7cuC8Pib/eWDLGQKO/iDFi3/Nu0oW01F1RtzViIgMqP6cfVQbnWm00cw2mNnPzax2\nMIrLSdPeCcWVfLLqRR5atIE3tujhOyJy6OhP89EdwAOEm+CNA34djctPqSI48r0cUf84ZdbCHX9Z\nEXdFIiIDpj+hUOPud7h7e9TdCdRkua7cdsxcEu1NXJO6jx/+ZSVbdrbEXZGIyIDoTyhsNrPLzCwZ\ndZcBW7JdWE4bPxtGzWTuqDdwnO8/qb0FETk09CcUPgpcDKwH1gHvB67IZlE5zwxO/BjFm1/l6mnb\nuetp7S2IyKFhn6Hg7m+4+/nuXuPuo9z9QuCiQagttx39QSgs5RMlj7CrtYN336TTU0Vk6DvQG+J9\ndkCrGIqKyuCYuZQve4CPH1PClsZWVm5ujLsqEZGDcqChYANaxVB1yj9Aup1ryh+mIJngK79bHHdF\nIiIH5UBDQfd3ABg5BWZewPBX7qK2pJ0/LNzA40v1ZDYRGbr6DIU+bpndYGY7CNcs7JOZnW1mS8xs\nmZld28v0/zazl6NuqZnVH8TvEo+3fApaGvht8ReZXD2cf73/VZrbOuKuSkTkgPQZCu5e5u7lvXRl\n7r7P22OYWRK4GTgHmAnMNbOZPdZxjbsf6+7HAv8D/OLgfp0YjDseDj+TgtbtfOXdk3hj6y5ufPi1\nuKsSETkg2Xzy2mxgmbsvd/dWwvMXLtjL/HOBe7JYT/ac/s/QtJWTN97HxbNqueXx13n69fy+lENE\nhqZshsI4YHXGcF00bg9mNhGYDDzSx/QrzWy+mc3ftCkH2+zHnQAlI+Gxr/BvZ4xhctVwrvnpy2xr\nbI27MhGR/ZLNUOjtDKW+DlBfAvzM3XttjHf3W919lrvPqqnJ0TtsXP5bwBn+zDe5ae5xbGho5vRv\nPqZnLojIkJLNUKgDxmcM1wJr+5j3EoZq01Gn0TPh+I/A87dxVNFGvviemdTvauOup1fFXZmISL9l\nMxSeB6aZ2WQzKyR88D/QcyYzm0F4rOfTWaxlcJz+z+AOt53JR98ykcqSAq57YCHPrdgad2UiIv2S\ntVBw93bCk9n+ACwG7nX3hWZ2vZmdnzHrXGCeHwrtLKWj4OyvQHM99spPefyfTmdy9XCu+vELrN6q\n5y6ISO6zofZZPGvWLJ8/f37cZfQtnYY7zoFNf4NPPsfy5uGc9d9PkEoYT3z+dEaXF8ddoYjkITN7\nwd1n7Wu+bDYf5adEAi74DjRvh++ewpSaUn521SmkEsaltz2ru6mKSE5TKGRD9TQ440uwawss+hXH\nTRjB7ZefSN22XVz2g+fYvqst7gpFRHqlUMiWt3wKCofDzz4GOzZw0pQqbv3wLP62roFTvvow67Y3\nxV2hiMgeFArZkkzBxx+GRBLu/3tIpzl1eg0//OhsEmZcePNTLFy7Pe4qRUR2o1DIplFHwLtugNcf\nhpuOBeDU6TXcd9UpJMz4wC1P8/DiDTEXKSLSTaGQbbM+BkdeBPVvwJLfAXDEmHLu/+RbAfjYD+dz\n/a8X6c6qIpITFArZZgYX3ByOL8y7FDYsBGB0eTEv/utZfOSUidz+1AqOu/6P/G19Q8zFiki+UygM\nhsJhcPXzMLwG7rkEGjcDUFyQ5N8vOIo7rjiR9nSac779JN/+01Ja2rXXICLxUCgMlvKxcMlPYHsd\n3HgsNHfvFZw+YxRPf+EMzjtmLN/+02sc8+8P6dYYIhILhcJgqj0BPvhjaG+Cuz8ALTu7JlWXFnHT\n3OO484oTqS4t4uLvPc0JX/6jbo8hIoNKoTDY3vRueN8PYPUz8K0joLVxt8lzZozioWtO5erTD2fr\nrlZO/fqjfGbeS7y6Rqevikj26d5Hcfnrz+AXn4CJbw17DyWVe8yybnsTtz25gjueWkHa4ZQpVVx5\n6hROm15DItHb4ypERHrX33sfKRTi9Mp94cK2qqlw6X1QOaHX2bY3tTHvuTe446mVrG9oprggwXXn\nHcl5x4xleNE+H5ctIqJQGDJWPAF3XQiWgI/9ITzasw9tHWl++8o6bn1iOYvWNZAwOO+YsVx47Dje\nNq2agqRaA0WkdwqFoWTTErjl7ZBugwtvgWM+uNfZ3Z35q7bxy5fW8NtX1rG9qY1Uwpg7ewJnzRzN\nSVNGUpRKDlLxIjIUKBSGmp0b4b7LYdVTMPtKeOcNkCrc58ta29M8vnQT97+8hocXb6C5LU3C4J0z\nD+MdR4xizowaRpXpGQ4i+U6hMBR1tMGfroOnvwO1s+EDd0BFbb9f3tzWwV9e38yfFm/kkcUbWd/Q\nDMCUmuHMnjSSk6aM5G2H11BTVpSlX0BEcpVCYShbeD/87KPhFhnvvwNmnr/v1/Tg7ixc28BTyzbz\n3IqtPLZ0Ex3p8F6/eVwFc2bUcNyESo4cW8GosiLMdDaTyKFMoTDUbXkdvvf2cB3D9LPDs59HTjng\nxaXTzqJ1DTy2ZCOPLtnEC6u2dU2rLi1k5tgKjhpbzpFjKzhybDkTRg7Taa8ihxCFwqGgvRWe+x78\n8d/A03Dq5+Btnw33UjpIO5rbWLxuB6+u2c7CtQ387tV1NLV20PnXUFqUYuaYcmaOLeeocSEoDh9V\nqjOcRIYohcKhpGFd2Gto3AQV4+Fd/wlHnBealwZQc1sHr23YycK123l1bQiLxesaaG5LA1CYSjBj\ndBlHji3nyLHlTB1VytSaUjU/iQwBCoVD0cqn4O73Q9sumHI6nHkdjD02q6vsSDsrNu9k4doGFq5t\n6Nqz2N7U/Zzp4YVJptSUMqVmOFOjn1OqSxlVXsSIYYUk1QwlEjuFwqGqox3m/wD+8AVId8C0d8Fp\nn4fafb7XA8bdWbu9meWbdrJicyPLNzXy+qadLN/UyJr63Z89XZhMMLFqGLUjSjisopjR5aE7rLyz\nPwSHjl+IZJdC4VDXvB2euxWevhmatkFxJcydBxNOHvBmpf3R1NrBis2NrNjcyOadLazd3sTyTY2s\nrW9iQ0Mzm3e27vGagqQxqqyYUeVFVA0vorq0kJHDC6kqLaJqeCFVpYWMGFZIRUkB5SUFlBWlFCIi\n+0mhkC9adoY9h4e/HK6IHncCnPwPMPMCSBbEXd0eWtvTbNrZwvrtzWxsaGZ9QzMbGlrY2NDMhh3N\nbNnZypbGVrY2tnadQtuTGZQXF1BRsntXXrLnuJ5dWbECRfKTQiHftO6Cl++Gh/41PK+hbCzM/jic\ncAUMGxl3dfstnXYamtvYvDMERP2uVrY3te21a4h+tnX0/TdtBmVFKSqG9RYahbsNDytKUlIQdYXh\nZ3FGv46VyFCiUMhX6TQs+yM887+w/DHAwl7DsR+CqWdA8tC+q6q709TW0R0Wu3oPjvoDCJSeClOJ\n3UKjuCBJSUEiCo1U9DPMU1y4Z8CUZIzrnF6YSlCYTFCQTFCQNAqi4cJkQns4clByIhTM7GzgRiAJ\n3ObuX+1lnouB6wAHFrj7h/a2TIXCftiwCF78ITx/G6TbQ3PSKVfD8X93UBfCHap6BkpTa0fo2qKu\ntYPmrv509LM9mp7uMT3078p4fWt7+qDqSyaMgqSFkEh1Bkd3f2HSQqh0DSe6hotSiT1eV5A0CpIJ\nUtFrUz3Gh2nWFVLhtRYtK0lBKkxLJROkEkYqaRQkFF65KvZQMLMksBQ4C6gDngfmuvuijHmmAfcC\n73D3bWY2yt037m25CoUD0N4Krz0EL/0Ylv4ujJt8Khz/kXC9Q0r3QhoMHWnfa2g0t3bQ2pGmrcNp\n60jT1pGmtT0dxrV3j2tpT3f1t3V4ND3M19re/ZrO/pYew20dadr7OF4zEBIGqUQIlFQiBEwyYV0h\nk0oYyYSRTCQy+qPOjEQCEhZY8hMLAAARaUlEQVSGE2ZRfxiXiMYlDRLR/KlkmDeVSEQ/w3ydy071\nWFf3/D3G96ila3nJ3adnrqezCbHnx6gZGfXTVfce/dbdn0xYVq/36W8oZLMtYTawzN2XRwXNAy4A\nFmXM8wngZnffBrCvQJADlCqEI94Tuoa18NLd8MQ3wrMcEkl48wdDE9PU0xUQWZRMGMOLUjnxYCR3\n7wqf9ihY2tNR+KTTu41va+8Oqs5A6gyXzhDrSIef7R0eltPhtEfh054Oy2rr6O7vSDsdHn62p8Pr\nO8e3Rj/dO+cJ9Xa+xj0EbEfaSXeO71pOWEdH2verKTDXGKHppHOnyzAwmDhyGI98bk5W153Nv85x\nwOqM4TrgpB7zTAcws6cITUzXufvvey7IzK4ErgSYMKH3p5NJP5WPhdP+Cd7+j7D80fBY0L/+FBb8\nBCwJR70vBMThZ0BBSdzVSpaYGYWp0Nx0KEv3DJ+O7tBoT/cIpbR3BVN7FDjdw328psO7vt13fsd3\nIO0h1NJRgGX2pzuDzTP6o/FpDzV39btHw6H/3UePyfo2y2Yo9LYf1DO6U8A0YA5QCzxpZke5e/1u\nL3K/FbgVQvPRwJeahxKJ8MF/+Blw3o1hr2HR/bDgHvjrveFJcCUjwnMdpr0ThlfFXbHIfkskjARG\ngZ451W/ZDIU6YHzGcC2wtpd5nnH3NmCFmS0hhMTzWaxLekoVwrQzQ/ee/4aVf4bFD4RjEPdfFeYp\nKg9XTs84NzxTWkQOSdk80JwiHGg+A1hD+KD/kLsvzJjnbMLB54+YWTXwEnCsu2/pa7k60DyI3GHt\nS7Dkd+HK6bbGMD5VAif9nxAQtbPCcQkRyWmxn30UFXEu8G3C8YLb3f0GM7semO/uD1hojPsmcDbQ\nAdzg7vP2tkyFQoy2rYKlv4clD8LyxwGHRAEc/cGwlzHp7TC8Ou4qRaQXOREK2aBQyBHN2+G1P4a9\niIW/BO8I40cfBZNPC6e8TnwLFJfHW6eIAAoFGUwd7aGZacXj8JeboGVHeCgQQGEZnHRlCInxJ+mM\nJpGYKBQkPm3NUPdcOKNp+eOhHwCDSW8LXe2J4XhEcUWspYrkC4WC5I6WHbDq6bAnMf+O7gPWAAXD\n4LgPd4fFELx5n8hQoFCQ3NXcAGvmw+rn4Y2/hD2Kzuam0UeFZ0KMPR7GHgc1M3R2k8gAyIXbXIj0\nrrgcpr4jdBDuzbT2RVj5JPzlf2D+YvDbwjRLQGFp6M769xAUI6eGi+9EZMBpT0FyTzoNW5aFg9dr\nXwrPiWhp6J5uyRASRaXwzi+HvYoRk2J94pxIrlPzkRxaOtph85LuoFgwD1p37j5PcSXM+mjYmxh7\nHFTUKihEIgoFOfS1t8KmxbDmRXjsKyEkWnfRfYstg5JKmH1ld1CUHRZnxSKxUShIfmprhg0LwzGK\nx78egqJtV/f0ZGFoejr5H6KgOFZXYUteUCiIdGpthPV/Dc1OT34TWnaG51h3ShZB4fDwRLqaGaGr\nng5FZfHVLDLAFAoie9PcAOsWhKB46kZo2ho9Pivz/8HCxXXHzM0Iixm6jbgMSQoFkf3V0QbbVsKm\nJbDpb/DsLaHpqa2p+zoKCDcBLCgJF9697RqomQ41b4KyMTqwLTlLoSAyUNJpaKiLwiIKjEX3hyu1\nM1kyBEXhMHjrp0NQjDpCYSE5QaEgkm3usHNjOFW2MyxeuS/sXaTbuuez6IrsYSPhxI+Hi++qpoSf\nJZXx1C55R6EgEqfGzbBxcQiKJ/4LGjdBIgUdLbvPl0iFpqhUcTh1duQUqDo8PN1OB7plACkURHJR\nWzNsWwFbXoetr8PW5fDqL6Mrtnv+L1p4DOpRF0H1NKiaBtWHQ8UESOoONbJ/FAoiQ03rrigwloXu\nmVvCqbNtTZBuz5jRoKA4PBb1uEuj5qipYQ9Dxy+kDwoFkUNJ45YoLF4LP1+4C5q39XIaLeFgd0FJ\nCI3TvxDuCzViUggM3XE2bykURPJBOg0Na0JQbH0dnvhmtHfRvPsFegBY2IsoKg/HMFLF8M7rQ2BU\nTtRB70OcQkEk33W0w/Y3YNuqcP1F/Sp48UfQ3hy63ZqkCAe9PQ0lI+C4y7r3MConQsV4SBXG8EvI\nQFEoiMjeNW/vDoxtK+Hp/4XGjeH+UO3N7NEslSwKQTKsGk76BIyYHAJjxKRw/ygdy8hpCgUROXDp\nNOxY1x0Y9atg/u2wa2t4wFFH2+7zWyI0R3W0wvBR8LbPdAdG5YRwQZ/ESqEgItnTugvq3wiB8fsv\ndDdJNdeH6Zm3BYGw95EqCmdSlR4WQqNifHjmReX4cI8pySqFgojEwz1cvNe5l/HIl6MD380ZT9Dr\n8bljyRAaHa1R89SV3aFRURvOnEoWDPIvcmhRKIhIbkqnw7GL7XWwfTX86Tpobwldc31oiup5EBzC\n3ka6A0pGhuszKmp3D47iCh3X2AuFgogMXa2NsH1NCI0HPx9uD9LeAru2hD2G9hZ63dvAoagCjrww\nNEtpb6NLf0NB18qLSO4pHB7dknw6fOqFPaen0+F+Utvr4P5/iEKjOYxLt8JLP+p9bwODotJwJlWq\nKNz6vDM0KsZrb4Ms7ymY2dnAjUASuM3dv9pj+uXAN4A10ajvuPtte1um9hREpF9ad4UL+7avDuHx\n+Ndhx/pwJlR7FCK9KRjWHRqzP7H73kb52CG7txF785GZJYGlwFlAHfA8MNfdF2XMczkwy92v7u9y\nFQoiMiAy9zY6g+Ppm0NgdLSEJqw9blJI95lUyaLo2EZmaIwLF//l4N5GLjQfzQaWufvyqKB5wAXA\nor2+SkRkMCQSUDY6dLUnhHFv6fH9tHUXNKwNV4Z37m10hkbTVnjqJvY8tpEIgdHRCsOq4ISPhLDo\nDI2KcTl9Cm42Q2EcsDpjuA44qZf53mdmpxL2Kq5x99U9ZzCzK4ErASZMmJCFUkVEelE4LNyuvPrw\nMHz83+0+PZ2GXZvDnkb96tBc1bAWFsyDptZwNtUT39hzuV0Hxcth5vkhLDoDo7w2/CwcnvVfrzfZ\nbD76APAud/94NPxhYLa7/9+MeaqAne7eYmZXARe7+zv2tlw1H4nIkNLRFo5lNKwJexsNa+CZ74am\nq1Rx2PNIt/X+2oJh3U1Vsz8O08+Gw958QGXkQvNRHTA+Y7gWWJs5g7tvyRj8PvC1LNYjIjL4kgXh\n9NjKjI/Dt35693naW8IeRsOacCpuwxp49tbuU3FbdsAj/wEvz+v9bKwBlM1QeB6YZmaTCWcXXQJ8\nKHMGMxvj7uuiwfOBxVmsR0QkN6WKYOTk0HV6+2d3n6et563Qs1RKthbs7u1mdjXwB8Ipqbe7+0Iz\nux6Y7+4PAJ8ys/OBdmArcHm26hERGdIKSgZlNbqiWUQkD/T3mEJiMIoREZGhQaEgIiJdFAoiItJF\noSAiIl0UCiIi0kWhICIiXRQKIiLSZchdp2Bmm4BVB/jyamDzAJYz0FTfwVF9By/Xa1R9B26iu9fs\na6YhFwoHw8zm9+fijbiovoOj+g5erteo+rJPzUciItJFoSAiIl3yLRRujbuAfVB9B0f1Hbxcr1H1\nZVleHVMQEZG9y7c9BRER2QuFgoiIdMmbUDCzs81siZktM7Nrc6Ce8Wb2qJktNrOFZvbpaPx1ZrbG\nzF6OunNjrHGlmf01qmN+NG6kmf3RzF6Lfo6IqbYZGdvoZTNrMLPPxLn9zOx2M9toZq9mjOt1e1lw\nU/T3+IqZHR9Tfd8ws79FNfzSzCqj8ZPMrCljO94SU319vp9m9oVo+y0xs3fFVN9PM2pbaWYvR+MH\nffsNGHc/5DvCk99eB6YAhcACYGbMNY0Bjo/6y4ClwEzgOuBzcW+zqK6VQHWPcV8Hro36rwW+lgN1\nJoH1wMQ4tx9wKnA88Oq+thdwLvA7wICTgWdjqu+dQCrq/1pGfZMy54tx+/X6fkb/KwuAImBy9P+d\nHOz6ekz/JvCluLbfQHX5sqcwG1jm7svdvRWYB1wQZ0Huvs7dX4z6dxCeTz0uzpr66QLgh1H/D4EL\nY6yl0xnA6+5+oFe6Dwh3f4LwWNlMfW2vC4C7PHgGqDSzMYNdn7s/5O7t0eAzQG02a9ibPrZfXy4A\n5rl7i7uvAJYR/s+zZm/1mZkBFwP3ZLOGwZAvoTAOWJ0xXEcOfQCb2STgOODZaNTV0e787XE1z0Qc\neMjMXjCzK6Nxo919HYRgA0bFVl23S9j9nzFXth/0vb1y8W/yo4S9l06TzewlM3vczN4eV1H0/n7m\n2vZ7O7DB3V/LGJcr22+/5EsoWC/jcuJcXDMrBX4OfMbdG4DvAlOBY4F1hF3SuLzV3Y8HzgE+aWan\nxlhLr8ysEDgfuC8alUvbb29y6m/SzP4FaAfujkatAya4+3HAZ4GfmFl5DKX19X7m1PYD5rL7F5Nc\n2X77LV9CoQ4YnzFcC6yNqZYuZlZACIS73f0XAO6+wd073D0NfJ8s7xLvjbuvjX5uBH4Z1bKhs5kj\n+rkxrvoi5wAvuvsGyK3tF+lre+XM36SZfQR4D3CpRw3iUbPMlqj/BUKb/fTBrm0v72cubb8UcBHw\n085xubL9DkS+hMLzwDQzmxx9s7wEeCDOgqI2yB8Ai939WxnjM9uV3wu82vO1g8HMhptZWWc/4YDk\nq4Tt9pFoto8Av4qjvgy7fUPLle2Xoa/t9QDwd9FZSCcD2zubmQaTmZ0N/D/gfHfflTG+xsySUf8U\nYBqwPIb6+no/HwAuMbMiM5sc1ffcYNcXORP4m7vXdY7Ile13QOI+0j1YHeFsj6WExP6XHKjnbYTd\n3VeAl6PuXOBHwF+j8Q8AY2Kqbwrh7I4FwMLObQZUAQ8Dr0U/R8a4DYcBW4CKjHGxbT9COK0D2gjf\nZD/W1/YiNH/cHP09/hWYFVN9ywht851/g7dE874vet8XAC8C58VUX5/vJ/Av0fZbApwTR33R+DuB\nq3rMO+jbb6A63eZCRES65EvzkYiI9INCQUREuigURESki0JBRES6KBRERKSLQkFERLooFET6wcyO\n7XHb5vNtgG7BbuGW38MGYlkiB0vXKYj0g5ldTrjA7OosLHtltOzN+/GapLt3DHQtItpTkENK9HCT\nxWb2fQsPL3rIzEr6mHeqmf0+ugvsk2b2pmj8B8zsVTNbYGZPRLdGuR74YPTAlA+a2eVm9p1o/jvN\n7LsWHpq03MxOi+7oudjM7sxY33fNbH5U179H4z4FjAUeNbNHo3FzLTzc6FUz+1rG63ea2fVm9ixw\nipl91cwWRXcQ/a/sbFHJO3FfUq1O3UB2hIebtAPHRsP3Apf1Me/DwLSo/yTgkaj/r8C4qL8y+nk5\n8J2M13YNE25zMI9w64oLgAbgzYQvXS9k1NJ5i4sk8BhwdDS8kuhhRoSAeAOoAVLAI8CF0TQHLu5c\nFuH2DpZZpzp1B9tpT0EORSvc/eWo/wVCUOwmumX5W4D7okcofo/wNDyAp4A7zewThA/w/vi1uzsh\nUDa4+1893NlzYcb6LzazF4GXgCMJTw/r6UTgMXff5OHhN3cTnvgF0EG4qy6E4GkGbjOzi4BdeyxJ\n5ACk4i5AJAtaMvo7gN6ajxJAvbsf23OCu19lZicB7wZeNrM95tnLOtM91p8GUtGdPD8HnOju26Jm\npeJeltPbcwI6NXt0HMHd281sNuGpc5cAVwPv6EedInulPQXJSx4eaLTCzD4A4VbmZnZM1D/V3Z91\n9y8Bmwn37d9BeJb2gSoHGoHtZjaa8ByITpnLfhY4zcyqo1svzwUe77mwaE+nwt0fBD5DeAiNyEHT\nnoLks0uB75rZF4ECwnGBBcA3zGwa4Vv7w9G4N4Bro6amr+zvitx9gZm9RGhOWk5ooup0K/A7M1vn\n7qeb2ReAR6P1P+juvT2zogz4lZkVR/Nds781ifRGp6SKiEgXNR+JiEgXNR/JIc/Mbgbe2mP0je5+\nRxz1iOQyNR+JiEgXNR+JiEgXhYKIiHRRKIiISBeFgoiIdPn/RQk8tXXG9fEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xf658f98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "\n",
    "\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "\n",
    "test_std = cvresult['test-mlogloss-std']\n",
    "\n",
    "\n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "\n",
    "train_std = cvresult['train-mlogloss-std']\n",
    "\n",
    "\n",
    "x_axis = range(0,cvresult.shape[0])\n",
    "\n",
    "pyplot.errorbar(x_axis,test_means,yerr=test_std,label='Test')\n",
    "\n",
    "pyplot.errorbar(x_axis,train_means,yerr=train_std,label='Train')\n",
    "\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "             \n",
    "pyplot.xlabel(\"n_estimators\")         \n",
    "             \n",
    "pyplot.ylabel(\"Log Loss\")     \n",
    "             \n",
    "pyplot.savefig('n_estimators.png')\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 根据第一步获取的弱学习器个数n_estimators = 191，,其他参数默认，再调整树的深度：max_depth\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [4, 6, 8]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = range(4,10,2)\n",
    "\n",
    "param_test= dict(max_depth=max_depth)\n",
    "\n",
    "param_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=1, missing=None, n_estimators=191,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.3),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'max_depth': [4, 6, 8]}, pre_dispatch='2*n_jobs',\n",
       "       refit=True, return_train_score='warn', scoring='neg_log_loss',\n",
       "       verbose=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#params = {\"objective\": \"multi:softprob\", \"eval_metric\":\"mlogloss\", \"num_class\": 9}\n",
    "xgb2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=191,  #根据第一步获取的弱学习器个数n_estimators = 191\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "gearch2_1 = GridSearchCV(xgb2,param_grid=param_test,scoring='neg_log_loss',n_jobs=-1,cv=3)\n",
    "\n",
    "gearch2_1.fit(train_X,train_y)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 获取树的最大深度，为6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Intstalled\\Anaconda2\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.60253, std: 0.00492, params: {'max_depth': 4},\n",
       "  mean: -0.60127, std: 0.00530, params: {'max_depth': 6},\n",
       "  mean: -0.61242, std: 0.00286, params: {'max_depth': 8}],\n",
       " {'max_depth': 6},\n",
       " -0.60127129299414173)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gearch2_1.grid_scores_,gearch2_1.best_params_,gearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 然后再调整min_child_weight，前提是 弱学习器个数和最大深度确定。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'min_child_weight': [1, 3, 5]}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_child_weight_test = range(1,6,2)\n",
    "\n",
    "param_test3 = dict(min_child_weight =min_child_weight_test )\n",
    "\n",
    "param_test3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 根据参数重新训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#params = {\"objective\": \"multi:softprob\", \"eval_metric\":\"mlogloss\", \"num_class\": 9}\n",
    "xgb3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=191,  #根据第一步获取的弱学习器个数n_estimators = 191\n",
    "        max_depth=6,#根据前一个步骤获取的最大深度\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "gearch3_1 = GridSearchCV(xgb3,param_grid=param_test3,scoring='neg_log_loss',n_jobs=-1,cv=3)\n",
    "\n",
    "gearch3_1.fit(train_X,train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 将测试结果保存到文件里"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "XGBoostError",
     "evalue": "need to call fit beforehand",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m\u001b[0m",
      "\u001b[1;31mXGBoostError\u001b[0mTraceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-58f914cb6643>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0my_test_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mxgb2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtest_X\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mout_df1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_test_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Intstalled\\Anaconda2\\lib\\site-packages\\xgboost-0.7-py2.7.egg\\xgboost\\sklearn.pyc\u001b[0m in \u001b[0;36mpredict_proba\u001b[1;34m(self, data, output_margin, ntree_limit)\u001b[0m\n\u001b[0;32m    534\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpredict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutput_margin\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mntree_limit\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    535\u001b[0m         \u001b[0mtest_dmatrix\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mDMatrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmissing\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmissing\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnthread\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 536\u001b[1;33m         class_probs = self.get_booster().predict(test_dmatrix,\n\u001b[0m\u001b[0;32m    537\u001b[0m                                                  \u001b[0moutput_margin\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutput_margin\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    538\u001b[0m                                                  ntree_limit=ntree_limit)\n",
      "\u001b[1;32mC:\\Intstalled\\Anaconda2\\lib\\site-packages\\xgboost-0.7-py2.7.egg\\xgboost\\sklearn.pyc\u001b[0m in \u001b[0;36mget_booster\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    177\u001b[0m         \"\"\"\n\u001b[0;32m    178\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_Booster\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 179\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mXGBoostError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'need to call fit beforehand'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    180\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_Booster\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    181\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mXGBoostError\u001b[0m: need to call fit beforehand"
     ]
    }
   ],
   "source": [
    "y_test_pred = xgb2.predict_proba(test_X)\n",
    "\n",
    "out_df1 = pd.DataFrame(y_test_pred)\n",
    "\n",
    "out_df1.columns = ['high','medium','low']\n",
    "\n",
    "out_df = pd.concat([out_df1],axis=1)\n",
    "\n",
    "out_df.to_csv(\"xgb_Rent.csv\",index=False)"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
